package com.cooper.java.scraper;

import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.NameValuePair;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.message.BasicNameValuePair;
import org.apache.http.util.EntityUtils;

import java.io.*;
import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/**
 * Created with IntelliJ IDEA.
 * User: cooperlee
 * Date: 11/15/12
 * Time: 9:49 AM
 */
public class HttpDownload {

    private static final String POST_URL_REG = "(.+)\\s#POST\\[(.+)\\]$";
    private static final String POST_PARAM_SPLIT_REG = "\"(.*?)\":\"(.*?)\"(,|$)";

    public HttpURLConnection createDefaultConn(String url)
            throws IOException {
        return createDefaultConn(url, null);
    }

    public HttpURLConnection createDefaultConn(String url,
                                               Map<String, String> headers) throws MalformedURLException,
            IOException {
        HttpURLConnection result = (HttpURLConnection) new URL(url)
                .openConnection();
        result.setConnectTimeout(params.getConnTimeout());
        result.setRequestMethod("GET");

        if (headers == null) {
            result.setRequestProperty(
                    "Accept",
                    "image/gif, image/jpeg, image/pjpeg, image/pjpeg, application/x-shockwave-flash, application/xaml+xml, application/vnd.ms-xpsdocument, application/x-ms-xbap, application/x-ms-application, application/vnd.ms-excel, application/vnd.ms-powerpoint, application/msword, */*");
            // result.setRequestProperty("Accept-Language", "zh-CN");
            result.setRequestProperty("Referer", url);

            // result.setRequestProperty("Charset", "UTF-8");
            result.setRequestProperty(
                    "User-Agent",
                    "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.2; Trident/4.0; .NET CLR 1.1.4322; .NET CLR 2.0.50727; .NET CLR 3.0.04506.30; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729)");
            result.setRequestProperty("Connection", "Keep-Alive");
        } else {
            for (Map.Entry<String, String> entry : headers.entrySet())
                result.setRequestProperty(entry.getKey(), entry.getValue());
        }

        return result;
    }

    protected DownloadParams params;

    public HttpDownload(DownloadParams params) {
        this.params = params;
    }

    public DownloadParams getParams() {
        return params;
    }

    public String getRedirectUrl(String oldUrl) {
        return getRedirectUrl(oldUrl, null);
    }

    public String getRedirectUrl(String oldUrl, Map<String, String> headers) {
        try {
            HttpURLConnection conn = createDefaultConn(oldUrl);
            try {
                conn.connect();
                if (conn.getResponseCode() == HttpURLConnection.HTTP_OK) {
                    return conn.getURL().toString();
                } else if (conn.getResponseCode() == HttpURLConnection.HTTP_MOVED_TEMP) {
                    return getRedirectUrl(conn.getHeaderField("Location"), headers);
                } else {
                    throw new IOException("ConnectError"
                            + conn.getResponseCode());
                }
            } finally {
                conn.disconnect();
            }
        } catch (Exception e) {
            return oldUrl;
        }
    }

    public boolean urlExists(String url) throws MalformedURLException,
            IOException, DownloadException {

        HttpURLConnection conn = (HttpURLConnection) new URL(url)
                .openConnection();
        try {
            conn.setConnectTimeout(params.getConnTimeout());
            conn.connect();
            switch (conn.getResponseCode()) {
                case HttpURLConnection.HTTP_CLIENT_TIMEOUT:
                case HttpURLConnection.HTTP_GATEWAY_TIMEOUT:
                    throw new DownloadException("timeout");
                case HttpURLConnection.HTTP_ACCEPTED:
                    System.out.println("url return:HTTP_ACCEPTED" );
                case HttpURLConnection.HTTP_OK:
                    System.out.println("url return:HTTP_OK" );
                    return true;
                default:
                    return false;
            }
        } finally {
            conn.disconnect();
        }
    }

    public boolean urlCanUse(String url) {
        try {
            return urlExists(url);
        } catch (Exception e2) {
            return false;
        }
    }

    private String downloadPage(String url, String encoding, int retry,
                                boolean useCache, Map<String, String> headers)
            throws DownloadException {
        try {


            String result;
            Pattern p = Pattern.compile(POST_URL_REG);
            Matcher m = p.matcher(url);
            if (m.find()) {
                String postUrl = m.group(1);
                String postParamStr = m.group(2);
                p = Pattern.compile(POST_PARAM_SPLIT_REG);
                m = p.matcher(postParamStr);
                List<NameValuePair> postParams = new ArrayList<NameValuePair>();
                while (m.find()) {
                    postParams.add(new BasicNameValuePair(m.group(1), m
                            .group(2)));
                }
                result = httpPostPage(postUrl, postParams, encoding);
            } else {
                result = httpGetPage(url, encoding, headers);
            }

            if ((result == null || "".equals(result.trim()))
                    && !Thread.currentThread().isInterrupted()) {

                throw new DownloadException(url);
            }


            return result;
        } catch (Exception e) {
            if (!Thread.currentThread().isInterrupted()) {
                if (retry <= params.getRetryCount()) {
                    return downloadPage(url, encoding, ++retry, useCache,
                            headers);
                } else {
                    e.printStackTrace();
                    throw new DownloadException(e);
                }
            } else
                return null;
        }
    }

    public String downloadPage(String url, String encoding)
            throws DownloadException {
        return downloadPage(url, encoding, 0, true, null);
    }

    public String downloadPage(String url) throws DownloadException {
        return downloadPage(url, null);
    }

    public String downloadPage(String url, String encoding, boolean useCache)
            throws DownloadException {
        return downloadPage(url, encoding, 0, useCache, null);
    }

    public String downloadPage(String url, boolean useCache)
            throws DownloadException {
        return downloadPage(url, null, 0, useCache, null);
    }

    public String downloadPage(String url, String encoding, boolean useCache,
                               Map<String, String> headers) throws DownloadException {
        return downloadPage(url, encoding, 0, useCache, headers);
    }

    private String httpPostPage(String url, List<NameValuePair> nvparams,
                                String encoding) throws ClientProtocolException, IOException {
        DefaultHttpClient httpClient = new DefaultHttpClient(
                params.createHttpParameters());
        try {
            HttpPost httpRequest = new HttpPost(url);
            httpRequest.setEntity(new UrlEncodedFormEntity(nvparams, encoding));
            HttpResponse httpResponse = httpClient.execute(httpRequest);
            if ((httpResponse.getStatusLine().getStatusCode() == 200)
                    && !Thread.currentThread().isInterrupted()) {
                return EntityUtils.toString(httpResponse.getEntity());
            } else {
                return null;
            }
        } finally {
            httpClient.getConnectionManager().shutdown();
        }
    }

    private String httpGetPage(String url, String encoding,
                               Map<String, String> headers) throws ClientProtocolException,
            IOException {
        DefaultHttpClient httpClient = new DefaultHttpClient(
                params.createHttpParameters());
        try {
            HttpGet request = new HttpGet(url);

            if (headers == null) {

                request.setHeader(
                        "User-Agent",
                        "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.2; Trident/4.0; .NET CLR 1.1.4322; .NET CLR 2.0.50727; .NET CLR 3.0.04506.30; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729)");
                // request.setHeader("User-Agent",
                // "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.1.3) Gecko/20090824 Firefox/3.5.3");
                request.setHeader("Accept",
                        "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
                // request.setHeader("Accept-Charset",
                // "ISO-8859-1,utf-8;q=0.7,*;q=0.7");

                request.setHeader("Referer", url);
            } else {
                for (Map.Entry<String, String> entry : headers.entrySet()) {
                    request.setHeader(entry.getKey(), entry.getValue());
                }
            }

            HttpResponse response = httpClient.execute(request);
            HttpEntity entity = response.getEntity();
            InputStream is = entity.getContent();
            if (is != null) {
                try {
                    InputStreamReader inputStreamReader;
                    if (encoding == null || "".equals(encoding)) {
                        inputStreamReader = new InputStreamReader(is);
                    } else {
                        inputStreamReader = new InputStreamReader(is, encoding);
                    }
                    BufferedReader in = new BufferedReader(inputStreamReader,
                            10 * 1024);
                    String str = "";
                    StringBuffer result = new StringBuffer("");
                    while ((str = in.readLine()) != null
                            && !Thread.currentThread().isInterrupted()) {
                        result.append(str);
                    }
                    if (!Thread.currentThread().isInterrupted()) {
                        return result.toString();
                    } else
                        return null;
                } finally {
                    is.close();
                }
            } else
                return null;
        } finally {
            httpClient.getConnectionManager().shutdown();
        }
    }


    public void downloadFile(String url, File file) throws DownloadException {
        downloadFile(url, file, 0);
    }

    private void downloadFile(String url, File file, int retry)
            throws DownloadException {
        try {
            httpGetFile(url, file);
        } catch (Exception e) {
            if (retry <= params.getRetryCount()) {
                downloadFile(url, file, ++retry);
            } else {
                throw new DownloadException(e);
            }
        }
    }

    private void httpGetFile(String url, File file)
            throws ClientProtocolException, IOException {
        DefaultHttpClient client = new DefaultHttpClient(
                params.createHttpParameters());
        try {
            HttpGet get = new HttpGet(url);
            HttpResponse response = client.execute(get);
            HttpEntity entity = response.getEntity();
            // long length = entity.getContentLength();
            InputStream is = entity.getContent();
            if (is != null) {
                FileOutputStream fileOutputStream = new FileOutputStream(file);
                try {
                    byte[] buf = new byte[1024];
                    int ch = -1;
                    while ((ch = is.read(buf)) != -1) {
                        if (!Thread.currentThread().isInterrupted()) {
                            fileOutputStream.write(buf, 0, ch);
                        } else if (file.exists()) {
                            file.delete();
                        }
                    }
                    if (!Thread.currentThread().isInterrupted())
                        fileOutputStream.flush();
                } finally {
                    is.close();
                    fileOutputStream.close();
                }
            }
        } finally {
            client.getConnectionManager().shutdown();
        }
    }
}
